{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "A100"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "fe0060c375664293be1d4759f8895f0d": {
          "model_module": "jupyter-webrtc",
          "model_name": "AudioRecorderModel",
          "model_module_version": "~0.6.0",
          "state": {
            "_data_src": "blob:https://me20u6ipsp-496ff2e9c6d22116-0-colab.googleusercontent.com/a4aa93b4-563a-46ef-b14a-2a0793a0bedb",
            "_dom_classes": [],
            "_model_module": "jupyter-webrtc",
            "_model_module_version": "~0.6.0",
            "_model_name": "AudioRecorderModel",
            "_view_count": null,
            "_view_module": "jupyter-webrtc",
            "_view_module_version": "~0.6.0",
            "_view_name": "AudioRecorderView",
            "audio": "IPY_MODEL_67104957e6274dcba6bf9ae5fbb9c73c",
            "autosave": false,
            "codecs": "",
            "filename": "record",
            "format": "webm",
            "layout": "IPY_MODEL_274a22b072ac4c22bfecab318e3e5581",
            "recording": false,
            "stream": "IPY_MODEL_016ef0369c6d41ef80a2c6833ae4d818"
          }
        },
        "67104957e6274dcba6bf9ae5fbb9c73c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "AudioModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "AudioModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "AudioView",
            "autoplay": true,
            "controls": true,
            "format": "webm",
            "layout": "IPY_MODEL_257f0df52c024353a5fdc02b4e11debf",
            "loop": true
          }
        },
        "274a22b072ac4c22bfecab318e3e5581": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "016ef0369c6d41ef80a2c6833ae4d818": {
          "model_module": "jupyter-webrtc",
          "model_name": "CameraStreamModel",
          "model_module_version": "~0.6.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "jupyter-webrtc",
            "_model_module_version": "~0.6.0",
            "_model_name": "CameraStreamModel",
            "_view_count": null,
            "_view_module": "jupyter-webrtc",
            "_view_module_version": "~0.6.0",
            "_view_name": "MediaStreamView",
            "constraints": {
              "audio": true,
              "video": false
            },
            "layout": "IPY_MODEL_edbe16a94aa94596a24763d5ad6bf996"
          }
        },
        "257f0df52c024353a5fdc02b4e11debf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "edbe16a94aa94596a24763d5ad6bf996": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers\n",
        "!pip install datasets\n",
        "!pip install ipywebrtc\n",
        "!pip install soundfile\n",
        "!pip install pydub\n",
        "!pip install ffmpeg-python"
      ],
      "metadata": {
        "id": "vQrvm2IxfP4G"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Automatic Speech Recognition"
      ],
      "metadata": {
        "id": "V0llYgTWT_39"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from transformers import pipeline\n",
        "from datasets import load_dataset\n",
        "import torchaudio\n"
      ],
      "metadata": {
        "id": "ydK86A8kbQcZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "def get_device():\n",
        "  device='cpu'\n",
        "  if torch.cuda.is_available():\n",
        "    device='cuda:0'\n",
        "  elif torch.backends.mps.is_available():\n",
        "    device='mps'\n",
        "  else:\n",
        "    device='cpu'\n",
        "  return device\n",
        "\n",
        "device = get_device()\n",
        "print (device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p0TVRNlQbMVa",
        "outputId": "0e86e3fb-b9e7-4406-b049-f65f26f197bc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cuda:0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vT5CS-uKUMku",
        "outputId": "256fc0bb-9536-4d6b-c552-c1d8f6916429"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Found cached dataset librispeech_asr_dummy (/Users/premtimsina/.cache/huggingface/datasets/hf-internal-testing___librispeech_asr_dummy/clean/2.1.0/d3bc4c2bc2078fcde3ad0f0f635862e4c0fef78ba94c4a34c4c250a097af240b)\n"
          ]
        }
      ],
      "source": [
        "\n",
        "\n",
        "\n",
        "pipe = pipeline(\n",
        "  \"automatic-speech-recognition\",\n",
        "  model=\"openai/whisper-large-v2\",\n",
        "  chunk_length_s=30,\n",
        "  device=device,\n",
        ")\n",
        "\n",
        "ds = load_dataset(\"hf-internal-testing/librispeech_asr_dummy\", \"clean\", split=\"validation\")\n",
        "sample = ds[0][\"audio\"]\n",
        "\n",
        "prediction = pipe(sample.copy(), batch_size=8)[\"text\"]\n",
        "\n",
        "# we can also return timestamps for the predictions\n",
        "prediction = pipe(sample.copy(), batch_size=8, return_timestamps=True)[\"chunks\"]\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(prediction)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x592a1rkUngO",
        "outputId": "c9d9c5cb-e9ec-4af8-d651-3752e0f6757f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[{'timestamp': (0.0, 5.44), 'text': ' Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.'}]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import output\n",
        "output.enable_custom_widget_manager()"
      ],
      "metadata": {
        "id": "EqwRGoTifn0N"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from ipywebrtc import CameraStream, AudioRecorder\n",
        "\n",
        "# Create camera stream\n",
        "camera = CameraStream(constraints={'audio': True, 'video': False})\n",
        "\n",
        "# Create audio recorder\n",
        "recorder = AudioRecorder(stream=camera)\n",
        "\n",
        "# Display recorder\n",
        "display(recorder)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107,
          "referenced_widgets": [
            "fe0060c375664293be1d4759f8895f0d",
            "67104957e6274dcba6bf9ae5fbb9c73c",
            "274a22b072ac4c22bfecab318e3e5581",
            "016ef0369c6d41ef80a2c6833ae4d818",
            "257f0df52c024353a5fdc02b4e11debf",
            "edbe16a94aa94596a24763d5ad6bf996"
          ]
        },
        "id": "78FAHuqpi0OH",
        "outputId": "b0963038-f8f3-4d51-d527-e609040d94d5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "AudioRecorder(audio=Audio(value=b'', format='webm'), stream=CameraStream(constraints={'audio': True, 'video': …"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fe0060c375664293be1d4759f8895f0d"
            }
          },
          "metadata": {
            "application/vnd.jupyter.widget-view+json": {
              "colab": {
                "custom_widget_manager": {
                  "url": "https://ssl.gstatic.com/colaboratory-static/widgets/colab-cdn-widget-manager/b3e629b1971e1542/manager.min.js"
                }
              }
            }
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import ffmpeg\n",
        "# Save the recording to a file\n",
        "recorder.save('output.webm')\n",
        "# Convert webm to wav\n",
        "ffmpeg.input('output.webm').output('output.wav').run()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x3SbEgyHknMt",
        "outputId": "28f4711f-2d9e-47b9-eb93-19936295942a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(None, None)"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torchaudio\n",
        "import torchaudio.transforms as T\n",
        "\n",
        "waveform, sample_rate = torchaudio.load('output.wav')\n",
        "\n",
        "# If audio is stereo, convert to mono\n",
        "if waveform.shape[0] > 1:\n",
        "    waveform = waveform.mean(dim=0)\n",
        "\n",
        "# Resample the waveform to 16kHz\n",
        "resampler = T.Resample(orig_freq=sample_rate, new_freq=16000)\n",
        "waveform = resampler(waveform)\n",
        "\n",
        "# Squeeze the tensor to remove the channel dimension\n",
        "waveform = waveform.squeeze()\n",
        "\n",
        "# Convert tensor to numpy array\n",
        "waveform_numpy = waveform.numpy()"
      ],
      "metadata": {
        "id": "ncSzB5TcvgSd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pipe = pipeline(\n",
        "  \"automatic-speech-recognition\",\n",
        "  model=\"openai/whisper-large-v2\",\n",
        "  chunk_length_s=30,\n",
        "  device=device,\n",
        ")\n",
        "prediction = pipe(waveform_numpy, batch_size=8)[\"text\"]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K058S5L1exCj",
        "outputId": "d4436f42-0870-46f6-fd6d-7dda7c57c1a9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py:1346: UserWarning: Using `max_length`'s default (448) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(prediction)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ehoZf6RWmkx-",
        "outputId": "59364abe-e02e-4820-d37a-af5369264fca"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " I live in New York City. Today is very interesting day.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "hGWw5PiWtczZ"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}